const fs = require('fs')
const Nightmare = require('nightmare');

const nightmare = Nightmare({
    waitTimeout: 60 * 1000 * 2,
    show: true,
    openDevTools: {
        mode: 'detach'
    },
});

const visitedUrls = new Set();
const crawlPage = async (url) => {
    if (visitedUrls.has(url)) {
        return;
    }
    visitedUrls.add(url);
    console.log(`Crawling ${url}...`);
    try {
        const page = nightmare.goto(url);
        const pageTitle = await page.title();
        await page.wait(2000)
        await page.inject('js', 'node_modules/jquery/dist/jquery.min.js')
        const {links, data} = await page.evaluate(() => {
            const linkNodes = document.querySelectorAll('a.crawler');
            const links = [];
            linkNodes.forEach((node) => {
                if (!node.href.baseVal)
                    links.push(node.href)
                else {
                    const link = `${window.location.href.split('#')[0]}${node.href.baseVal}`
                    links.push(link)
                }
            });
            return {links};
        });
        for (let link of links) {
            await crawlPage(link);
        }
    } catch (error) {
        console.error(`Failed to crawl ${url}: ${error}`);
    }
};


nightmare.goto('http://localhost:1999/#/login?redirect=aHR0cDovL2xvY2FsaG9zdDoxOTk5LyMvaG9tZQ%3D%3D')
    .type('input[placeholder="用户名"]', 'yanqi.xu')
    .type('input[placeholder="密码"]', '123123123')
    .click('button')
    .wait('.banner')
    .inject('js', 'node_modules/jquery/dist/jquery.min.js')
    .url()
    .then(async url => {
        await crawlPage(url)
        const urls = Array.from(visitedUrls)
        console.log(urls);
        const newStr = urls.join('\n')
        fs.writeFile('./urls.txt',newStr,function(err){
            if (err) {
                return console.log(err+'url写入失败的');
            }
            console.log('url写入成功');
        })
        nightmare.end();
    })
    .catch((error) => {
        console.error('Search failed:', error);
        nightmare.end();
    });



